import pathlib
import numpy as np
import re
from pathlib import Path
#import glob
import pathlib
import numpy as np
#filepath = 'drive/My Drive/MaEducation/access.log-20191119'
src = 'log/'
ext= '*.log*'
# only for test the codes
#line = '''27.17.18.210 - - [18/Nov/2019:13:40:23 +0800] "GET /xampp/phpmyadmin/index.php HTTP/1.1" 502 575 "-" "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0)" "-"
#"'''
#line = '''27.17.18.210 - - [18/Nov/2019:13:40:23 +0800] "GET /xampp/phpmyadmin/index.php HTTP/1.1" 502 575 "-" "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0)" "-"'''

filepath = 'drive/My Drive/MaEducation/access.log-20191119'
pattern = '''(?P<remote>[\d.]{7,}) - - \[(?P<datetime>[\w/: +]+)\] \
"(?P<method>\w+) (?P<url>\S+) (?P<ptotocol>[\w\d/.]+)" \
(?P<status>\d+) (?P<length>\d+) ".+" "(?P<useragent>.+)" ".+"'''
regex = re.compile(pattern)

datatype_convertor = {
    'datatime':lambda x:datetime.datetime.strptime(x,'%d/%b/%Y:%H:%M:%S %z'),
    'status':int,
    'length':int
}

def extract(logline:str) -> dict:
  m = regex.match(logline)
  if m:
    #print(m.groups())
    return {k:datatype_convertor.get(k,lambda x:x)(v) for k,v in m.groupdict().items()}
  else:
    return None #Exception('No match, {}'.format(line))
    #continue

def load_file(filepath:str,encoding='utf-8'):
  with open (filepath,encoding = encoding) as f:
    for line in f:
      fields = extract(line)
      if fields:
        yield fields
      else:
        continue
def load_folder(*src,encoding='utf-8',ext='*.log',recursive=True):
  for x in (src):
    P = Path(x)
    print(P)
    if P.is_dir():
      if isinstance(ext,str):
        ext = [ext]
      else:
        ext = list(ext)
      for e in ext:
        files = P.rglob(e) if recursive else P.glob(e)
        for file in files:
          print(file)
          yield from load_file(str(file.absolute()),encoding=encoding)

    elif P.is_file():
      yield from load_file(str(P.absolute()),encoding=encoding)

logData = load_folder(src,encoding='utf-8',ext=ext,recursive=False)
logData = list(logData)
temp_ip = []
temp_url = []
for iDict in logData:
  temp_ip.append(iDict['remote'])
  temp_url.append(iDict['url'])
temp_ip_uniq = set(temp_ip)
temp_url_uniq = set(temp_url)
ipDict = {}
urlDict = {}
for iDict in logData:
  if iDict['remote'] in temp_ip_uniq:
    ipDict[iDict['remote']] = 0 if iDict['remote'] not in ipDict.keys() else (ipDict[iDict['remote']] +1)
  if iDict['url'] in temp_url_uniq:
    urlDict[iDict['url']] = 0 if iDict['url'] not in urlDict.keys() else (urlDict[iDict['url']] +1)

maxIP = max(ipDict, key=ipDict.get)
maxIP_count = ipDict[maxIP]
maxUrl = max(urlDict,key=urlDict.get)
maxUrl_count = urlDict[maxUrl]
sorted_ipDict = sorted(ipDict,key=ipDict.get,reverse=True)
top10_IP = {} #records the top 10 visited ips
for i in sorted_ipDict[:10]:
  top10_IP[i] = ipDict[i]
sorted_urlDict = sorted(urlDict,key=urlDict.get,reverse=True)
top10_Url = {} # records the top 10 visited urls
for i in sorted_urlDict[:10]:
  top10_Url[i] = urlDict[i]
print('Task has been completed here!')

